{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jack/opt/anaconda3/envs/tf_2_0/lib/python3.7/site-packages/lightgbm/__init__.py:48: UserWarning: Starting from version 2.2.1, the library file in distribution wheels for macOS is built by the Apple Clang (Xcode_8.3.3) compiler.\n",
      "This means that in case of installing LightGBM from PyPI via the ``pip install lightgbm`` command, you don't need to install the gcc compiler anymore.\n",
      "Instead of that, you need to install the OpenMP library, which is required for running LightGBM on the system with the Apple Clang compiler.\n",
      "You can install the OpenMP library by the following command: ``brew install libomp``.\n",
      "  \"You can install the OpenMP library by the following command: ``brew install libomp``.\", UserWarning)\n"
     ]
    }
   ],
   "source": [
    "from deeptables.models import deeptable,deepnets\n",
    "from deeptables.utils import consts,dt_logging,batch_trainer\n",
    "from deeptables.datasets import dsutils\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Base dir:/Users/jack/opt/anaconda3/envs/tf_2_0/lib/python3.7/site-packages/deeptables/datasets\n"
     ]
    }
   ],
   "source": [
    "data = dsutils.load_adult()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 class detected, {' <=50K', ' >50K'}, so inferred as a [binary classification] task\n",
      "Start training DT model.['dnn_nets']\n",
      "train metrics:['AUC']\n",
      "eval metrics:['AUC', 'accuracy', 'recall', 'precision', 'f1']\n",
      "Fitting model...\n",
      "X_train.shape:(26048, 14),y_train.shape:(26048,)\n",
      "2 class detected, {' <=50K', ' >50K'}, so inferred as a [binary classification] task\n",
      "Preparing features cost:0.019079923629760742\n",
      "Imputation cost:0.08739614486694336\n",
      "Categorical encoding cost:0.14487409591674805\n",
      "Discretization cost:0.08411478996276855\n",
      "Extracting gbm features cost:1.2587871551513672\n",
      "fit_transform cost:1.6632287502288818\n",
      "Injected a callback [EarlyStopping]. monitor:val_AUC, patience:1, mode:max\n",
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (14)', 'input_continuous_all: (6)', 'input_continuous_gbm_leaves: (100)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [11, 18, 9, 17, 8, 7, 4, 43, 5, 14, 4, 5, 5, 5]\n",
      "output_dims: [4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 162)\n",
      "---------------------------------------------------------\n",
      "nets: ['dnn_nets']\n",
      "---------------------------------------------------------\n",
      "dnn: input_shape (None, 162), output_shape (None, 64)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 20838 samples, validate on 5210 samples\n",
      "Epoch 1/15\n",
      "20838/20838 [==============================] - 17s 820us/sample - loss: 0.3320 - AUC: 0.8954 - val_loss: 0.2957 - val_AUC: 0.9236\n",
      "Epoch 2/15\n",
      "20838/20838 [==============================] - 4s 198us/sample - loss: 0.2920 - AUC: 0.9203 - val_loss: 0.2836 - val_AUC: 0.9273\n",
      "Epoch 3/15\n",
      "20838/20838 [==============================] - 4s 184us/sample - loss: 0.2820 - AUC: 0.9259 - val_loss: 0.2824 - val_AUC: 0.9286\n",
      "Epoch 4/15\n",
      "20736/20838 [============================>.] - ETA: 0s - loss: 0.2742 - AUC: 0.9303Restoring model weights from the end of the best epoch.\n",
      "20838/20838 [==============================] - 4s 181us/sample - loss: 0.2740 - AUC: 0.9304 - val_loss: 0.2793 - val_AUC: 0.9290\n",
      "Epoch 00004: early stopping\n",
      "Model has been saved to:dt_output/dt_20200331 182923_dnn_nets/dnn_nets.h5\n",
      "Scoring...\n",
      "transform_X cost:0.8628330230712891\n",
      "predict_proba cost:1.7190001010894775\n",
      "\n",
      "------------['dnn_nets'] -------------Eval score:\n",
      "{'auc': 0.9120375442346713, 'accuracy': 0.860586519269154, 'recall': 0.5828981723237598, 'precision': 0.7685025817555938, 'f1': 0.6629547141796586}\n",
      "DT finished.\n",
      "DT - ['dnn_nets'] - done in 36s\n",
      "----------------------------------------------------------\n",
      "\n",
      "Start training DT model.['cross_nets']\n",
      "train metrics:['AUC']\n",
      "eval metrics:['AUC', 'accuracy', 'recall', 'precision', 'f1']\n",
      "Fitting model...\n",
      "X_train.shape:(26048, 14),y_train.shape:(26048,)\n",
      "2 class detected, {' <=50K', ' >50K'}, so inferred as a [binary classification] task\n",
      "Preparing features cost:0.018381595611572266\n",
      "Imputation cost:0.08629202842712402\n",
      "Categorical encoding cost:0.15778112411499023\n",
      "Discretization cost:0.0865330696105957\n",
      "Extracting gbm features cost:0.7112340927124023\n",
      "fit_transform cost:1.0911297798156738\n",
      "Injected a callback [EarlyStopping]. monitor:val_AUC, patience:1, mode:max\n",
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (14)', 'input_continuous_all: (6)', 'input_continuous_gbm_leaves: (100)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [11, 18, 9, 17, 8, 7, 4, 43, 5, 14, 4, 5, 5, 5]\n",
      "output_dims: [4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 162)\n",
      "---------------------------------------------------------\n",
      "nets: ['dnn_nets']\n",
      "---------------------------------------------------------\n",
      "dnn: input_shape (None, 162), output_shape (None, 64)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 20838 samples, validate on 5210 samples\n",
      "Epoch 1/15\n",
      "20838/20838 [==============================] - 15s 726us/sample - loss: 0.3273 - AUC: 0.8995 - val_loss: 0.3168 - val_AUC: 0.9074\n",
      "Epoch 2/15\n",
      "20838/20838 [==============================] - 4s 216us/sample - loss: 0.2892 - AUC: 0.9228 - val_loss: 0.2968 - val_AUC: 0.9170\n",
      "Epoch 3/15\n",
      "20838/20838 [==============================] - 5s 229us/sample - loss: 0.2779 - AUC: 0.9291 - val_loss: 0.2931 - val_AUC: 0.9192\n",
      "Epoch 4/15\n",
      "20838/20838 [==============================] - 6s 289us/sample - loss: 0.2724 - AUC: 0.9318 - val_loss: 0.2889 - val_AUC: 0.9212\n",
      "Epoch 5/15\n",
      "20480/20838 [============================>.] - ETA: 0s - loss: 0.2663 - AUC: 0.9354Restoring model weights from the end of the best epoch.\n",
      "20838/20838 [==============================] - 6s 268us/sample - loss: 0.2659 - AUC: 0.9356 - val_loss: 0.2898 - val_AUC: 0.9212\n",
      "Epoch 00005: early stopping\n",
      "Model has been saved to:dt_output/dt_20200331 182959_dnn_nets/dnn_nets.h5\n",
      "Scoring...\n",
      "transform_X cost:1.0262341499328613\n",
      "predict_proba cost:2.19035005569458\n",
      "\n",
      "------------['cross_nets'] -------------Eval score:\n",
      "{'auc': 0.9111616309076317, 'accuracy': 0.8608935974205435, 'recall': 0.6031331592689295, 'precision': 0.7561374795417348, 'f1': 0.6710239651416122}\n",
      "DT finished.\n",
      "DT - ['cross_nets'] - done in 41s\n",
      "----------------------------------------------------------\n",
      "\n",
      "Start training DT model.['dnn_nets']\n",
      "train metrics:['AUC']\n",
      "eval metrics:['AUC', 'accuracy', 'recall', 'precision', 'f1']\n",
      "Fitting model...\n",
      "X_train.shape:(26048, 14),y_train.shape:(26048,)\n",
      "2 class detected, {' <=50K', ' >50K'}, so inferred as a [binary classification] task\n",
      "Preparing features cost:0.022692203521728516\n",
      "Imputation cost:0.13198208808898926\n",
      "Categorical encoding cost:0.21001124382019043\n",
      "fit_transform cost:0.4013679027557373\n",
      "Injected a callback [EarlyStopping]. monitor:val_AUC, patience:1, mode:max\n",
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (8)', 'input_continuous_all: (6)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [11, 18, 9, 17, 8, 7, 4, 43]\n",
      "output_dims: [4, 8, 4, 8, 4, 4, 4, 8]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 50)\n",
      "---------------------------------------------------------\n",
      "nets: ['dnn_nets']\n",
      "---------------------------------------------------------\n",
      "dnn: input_shape (None, 50), output_shape (None, 64)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 20838 samples, validate on 5210 samples\n",
      "Epoch 1/15\n",
      "20838/20838 [==============================] - 16s 763us/sample - loss: 0.3761 - AUC: 0.8645 - val_loss: 0.5517 - val_AUC: 0.8633\n",
      "Epoch 2/15\n",
      "20838/20838 [==============================] - 4s 210us/sample - loss: 0.3360 - AUC: 0.8962 - val_loss: 0.3790 - val_AUC: 0.8970\n",
      "Epoch 3/15\n",
      "20838/20838 [==============================] - 3s 163us/sample - loss: 0.3282 - AUC: 0.9016 - val_loss: 0.3210 - val_AUC: 0.9127\n",
      "Epoch 4/15\n",
      "20352/20838 [============================>.] - ETA: 0s - loss: 0.3238 - AUC: 0.9044Restoring model weights from the end of the best epoch.\n",
      "20838/20838 [==============================] - 5s 251us/sample - loss: 0.3242 - AUC: 0.9045 - val_loss: 0.3149 - val_AUC: 0.9126\n",
      "Epoch 00004: early stopping\n",
      "Model has been saved to:dt_output/dt_20200331 183039_dnn_nets/dnn_nets.h5\n",
      "Scoring...\n",
      "transform_X cost:0.6480162143707275\n",
      "predict_proba cost:1.4889137744903564\n",
      "\n",
      "------------['dnn_nets'] -------------Eval score:\n",
      "{'auc': 0.9034084088727765, 'accuracy': 0.8516812528788577, 'recall': 0.5607049608355091, 'precision': 0.7456597222222222, 'f1': 0.6400894187779433}\n",
      "DT finished.\n",
      "DT - ['dnn_nets'] - done in 32s\n",
      "----------------------------------------------------------\n",
      "\n",
      "Start training DT model.['cross_nets']\n",
      "train metrics:['AUC']\n",
      "eval metrics:['AUC', 'accuracy', 'recall', 'precision', 'f1']\n",
      "Fitting model...\n",
      "X_train.shape:(26048, 14),y_train.shape:(26048,)\n",
      "2 class detected, {' <=50K', ' >50K'}, so inferred as a [binary classification] task\n",
      "Preparing features cost:0.014709949493408203\n",
      "Imputation cost:0.08382821083068848\n",
      "Categorical encoding cost:0.1444392204284668\n",
      "fit_transform cost:0.27435898780822754\n",
      "Injected a callback [EarlyStopping]. monitor:val_AUC, patience:1, mode:max\n",
      ">>>>>>>>>>>>>>>>>>>>>> Model Desc <<<<<<<<<<<<<<<<<<<<<<< \n",
      "---------------------------------------------------------\n",
      "inputs:\n",
      "---------------------------------------------------------\n",
      "['all_categorical_vars: (8)', 'input_continuous_all: (6)']\n",
      "---------------------------------------------------------\n",
      "embeddings:\n",
      "---------------------------------------------------------\n",
      "input_dims: [11, 18, 9, 17, 8, 7, 4, 43]\n",
      "output_dims: [4, 8, 4, 8, 4, 4, 4, 8]\n",
      "dropout: 0.3\n",
      "---------------------------------------------------------\n",
      "dense: dropout: 0\n",
      "batch_normalization: False\n",
      "---------------------------------------------------------\n",
      "concat_embed_dense: shape: (None, 50)\n",
      "---------------------------------------------------------\n",
      "nets: ['dnn_nets']\n",
      "---------------------------------------------------------\n",
      "dnn: input_shape (None, 50), output_shape (None, 64)\n",
      "---------------------------------------------------------\n",
      "stacking_op: add\n",
      "---------------------------------------------------------\n",
      "output: activation: sigmoid, output_shape: (None, 1), use_bias: True\n",
      "loss: binary_crossentropy\n",
      "optimizer: Adam\n",
      "---------------------------------------------------------\n",
      "\n",
      "Train on 20838 samples, validate on 5210 samples\n",
      "Epoch 1/15\n",
      "20838/20838 [==============================] - 17s 821us/sample - loss: 0.3836 - AUC: 0.8603 - val_loss: 0.5529 - val_AUC: 0.8699\n",
      "Epoch 2/15\n",
      "20838/20838 [==============================] - 5s 217us/sample - loss: 0.3356 - AUC: 0.8962 - val_loss: 0.3870 - val_AUC: 0.8870\n",
      "Epoch 3/15\n",
      "20838/20838 [==============================] - 4s 210us/sample - loss: 0.3293 - AUC: 0.9007 - val_loss: 0.3346 - val_AUC: 0.9049\n",
      "Epoch 4/15\n",
      "20838/20838 [==============================] - 4s 169us/sample - loss: 0.3248 - AUC: 0.9035 - val_loss: 0.3099 - val_AUC: 0.9120\n",
      "Epoch 5/15\n",
      "20608/20838 [============================>.] - ETA: 0s - loss: 0.3216 - AUC: 0.9060Restoring model weights from the end of the best epoch.\n",
      "20838/20838 [==============================] - 4s 178us/sample - loss: 0.3220 - AUC: 0.9059 - val_loss: 0.3129 - val_AUC: 0.9112\n",
      "Epoch 00005: early stopping\n",
      "Model has been saved to:dt_output/dt_20200331 183112_dnn_nets/dnn_nets.h5\n",
      "Scoring...\n",
      "transform_X cost:1.2425658702850342\n",
      "predict_proba cost:2.636363983154297\n",
      "\n",
      "------------['cross_nets'] -------------Eval score:\n",
      "{'auc': 0.905783753721059, 'accuracy': 0.8527560264087211, 'recall': 0.5652741514360313, 'precision': 0.7471958584987057, 'f1': 0.6436269044964698}\n",
      "DT finished.\n",
      "DT - ['cross_nets'] - done in 38s\n",
      "----------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "conf1 = deeptable.ModelConfig(\n",
    "    name='conf1',\n",
    "    apply_gbm_features=True,\n",
    "    gbm_feature_type='dense',\n",
    "    auto_discrete=True,\n",
    "    metrics=['AUC'],\n",
    ")\n",
    "\n",
    "conf2 = deeptable.ModelConfig(\n",
    "    name='conf2',\n",
    "    fixed_embedding_dim=False,\n",
    "    embeddings_output_dim=0,\n",
    "    apply_gbm_features=False, #*\n",
    "    gbm_feature_type='dense',\n",
    "    auto_discrete=False, #*\n",
    "    metrics=['AUC'],\n",
    "\n",
    ")\n",
    "bt = batch_trainer.BatchTrainer(data,'x_14',\n",
    "                                eval_size=0.2,\n",
    "                                validation_size=0.2,\n",
    "                                eval_metrics=['AUC','accuracy','recall','precision','f1'], #auc/recall/precision/f1/mse/mae/msle/rmse/r2\n",
    "                                verbose=1,\n",
    "                                dt_epochs=15,\n",
    "                                dt_config=[conf1,conf2],\n",
    "                                dt_nets=[['dnn_nets'],['cross_nets']],\n",
    "                               )\n",
    "ms = bt.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>type</th>\n",
       "      <th>*auc</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>recall</th>\n",
       "      <th>precision</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>conf1 - ['dnn_nets'] - eval</td>\n",
       "      <td>eval</td>\n",
       "      <td>0.912038</td>\n",
       "      <td>0.860587</td>\n",
       "      <td>0.582898</td>\n",
       "      <td>0.768503</td>\n",
       "      <td>0.662955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>conf1 - ['cross_nets'] - eval</td>\n",
       "      <td>eval</td>\n",
       "      <td>0.911162</td>\n",
       "      <td>0.860894</td>\n",
       "      <td>0.603133</td>\n",
       "      <td>0.756137</td>\n",
       "      <td>0.671024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>conf2 - ['cross_nets'] - eval</td>\n",
       "      <td>eval</td>\n",
       "      <td>0.905784</td>\n",
       "      <td>0.852756</td>\n",
       "      <td>0.565274</td>\n",
       "      <td>0.747196</td>\n",
       "      <td>0.643627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>conf2 - ['dnn_nets'] - eval</td>\n",
       "      <td>eval</td>\n",
       "      <td>0.903408</td>\n",
       "      <td>0.851681</td>\n",
       "      <td>0.560705</td>\n",
       "      <td>0.745660</td>\n",
       "      <td>0.640089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           model  type      *auc  accuracy    recall  \\\n",
       "0    conf1 - ['dnn_nets'] - eval  eval  0.912038  0.860587  0.582898   \n",
       "1  conf1 - ['cross_nets'] - eval  eval  0.911162  0.860894  0.603133   \n",
       "2  conf2 - ['cross_nets'] - eval  eval  0.905784  0.852756  0.565274   \n",
       "3    conf2 - ['dnn_nets'] - eval  eval  0.903408  0.851681  0.560705   \n",
       "\n",
       "   precision        f1  \n",
       "0   0.768503  0.662955  \n",
       "1   0.756137  0.671024  \n",
       "2   0.747196  0.643627  \n",
       "3   0.745660  0.640089  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ms.leaderboard()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
